{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'VERSION' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-4ce825ea6ee0>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      9\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mazureml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mestimator\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mEstimator\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m \u001b[0mVERSION\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'VERSION' is not defined"
     ]
    }
   ],
   "source": [
    "from azureml.core import Workspace, Experiment, Datastore\n",
    "from azureml.data.datapath import DataPath, DataPathComputeBinding\n",
    "from azureml.data.data_reference import DataReference\n",
    "from azureml.core.compute import ComputeTarget, AmlCompute\n",
    "from azureml.pipeline.core import Pipeline, PipelineData, PipelineParameter\n",
    "from azureml.pipeline.steps import PythonScriptStep, EstimatorStep\n",
    "from azureml.widgets import RunDetails\n",
    "from azureml.train.estimator import Estimator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Register/Reference a Datastore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "WARNING - Warning: Falling back to use azure cli login credentials.\nIf you run your code in unattended mode, i.e., where you can't give a user input, then we recommend to use ServicePrincipalAuthentication or MsiAuthentication.\nPlease refer to aka.ms/aml-notebook-auth for different authentication mechanisms in azureml-sdk.\n"
    },
    {
     "data": {
      "text/plain": "{'workspacefilestore': <azureml.data.azure_storage_datastore.AzureFileDatastore at 0x23b03aadd48>,\n 'workspaceblobstore': <azureml.data.azure_storage_datastore.AzureBlobDatastore at 0x23b2032e288>,\n 'smtprodwestus21globaluploadedresources': <azureml.data.azure_storage_datastore.AzureBlobDatastore at 0x23b20369408>,\n 'seer': <azureml.data.azure_storage_datastore.AzureBlobDatastore at 0x23b203698c8>}"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# workspace\n",
    "ws = Workspace.from_config()\n",
    "ws.datastores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data\n",
    "datastore = ws.datastores['seer']\n",
    "\n",
    "# compute target\n",
    "compute = ws.compute_targets['gandalf']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define Pipeline!\n",
    "The following will be created and then run:\n",
    "1. Pipeline Parameters (path on datastore)\n",
    "2. Data Prep Step\n",
    "3. Training Step\n",
    "4. Model Registration Step\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Pipeline Parameters\n",
    "We need to tell the Pipeline what it needs to learn to see!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "datapath = DataPath(datastore=datastore, path_on_datastore='burrito_tacos')\n",
    "data_path_pipeline_param = (PipelineParameter(name=\"data\", \n",
    "                                             default_value=datapath), \n",
    "                                             DataPathComputeBinding(mode='mount'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Data Process Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "seer_tfrecords = PipelineData(\n",
    "    \"tfrecords_set\",\n",
    "    datastore=datastore,\n",
    "    is_directory=True\n",
    ")\n",
    "\n",
    "prep = Estimator(source_directory='.',\n",
    "                      compute_target=compute,\n",
    "                      entry_script='prep.py',\n",
    "                      use_gpu=True,\n",
    "                      pip_requirements_file='requirements.txt')\n",
    "\n",
    "prepStep = EstimatorStep(\n",
    "    name='Data Preparation',\n",
    "    estimator=prep,\n",
    "    estimator_entry_script_arguments=[\"--source_path\", data_path_pipeline_param, \n",
    "                                      \"--target_path\", seer_tfrecords],\n",
    "    inputs=[data_path_pipeline_param],\n",
    "    outputs=[seer_tfrecords],\n",
    "    compute_target=compute\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Training Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "seer_training = PipelineData(\n",
    "    \"train\",\n",
    "    datastore=datastore,\n",
    "    is_directory=True\n",
    ")\n",
    "\n",
    "train = Estimator(source_directory='.',\n",
    "                      compute_target=compute,\n",
    "                      entry_script='train.py',\n",
    "                      use_gpu=True,\n",
    "                      pip_requirements_file='requirements.txt')\n",
    "\n",
    "trainStep = EstimatorStep(\n",
    "    name='Model Training',\n",
    "    estimator=train,\n",
    "    estimator_entry_script_arguments=[\"--source_path\", seer_tfrecords, \n",
    "                                      \"--target_path\", seer_training,\n",
    "                                      \"--epochs\", 5,\n",
    "                                      \"--batch\", 10,\n",
    "                                      \"--lr\", 0.001],\n",
    "    inputs=[seer_tfrecords],\n",
    "    outputs=[seer_training],\n",
    "    compute_target=compute\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Register Model Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "seer_model = PipelineData(\n",
    "    \"model\",\n",
    "    datastore=datastore,\n",
    "    is_directory=True\n",
    ")\n",
    "\n",
    "register = Estimator(source_directory='.',\n",
    "                      compute_target=compute,\n",
    "                      entry_script='register.py',\n",
    "                      use_gpu=True)\n",
    "\n",
    "registerStep = EstimatorStep(\n",
    "    name='Model Registration',\n",
    "    estimator=register,\n",
    "    estimator_entry_script_arguments=[\"--source_path\", seer_training, \n",
    "                                      \"--target_path\", seer_model],\n",
    "    inputs=[seer_training],\n",
    "    outputs=[seer_model],\n",
    "    compute_target=compute\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "outputs": [],
   "source": [
    "## Test Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": "WARNING - 'gpu_support' is no longer necessary; AzureML now automatically detects and uses nvidia docker extension when it is available. It will be removed in a future release.\nWARNING - 'gpu_support' is no longer necessary; AzureML now automatically detects and uses nvidia docker extension when it is available. It will be removed in a future release.\nWARNING - 'gpu_support' is no longer necessary; AzureML now automatically detects and uses nvidia docker extension when it is available. It will be removed in a future release.\n"
    }
   ],
   "source": [
    "pipeline1 = Pipeline(workspace=ws, steps=[prepStep, trainStep, registerStep])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": "Created step Data Preparation [d5c771b0][38d05de8-f403-433c-98d8-d3cd348c8305], (This step will run and generate new outputs)\nCreated step Model Training [8dc2ac8e][3e903d1d-4bd3-4132-a853-86b891cf7514], (This step will run and generate new outputs)\nCreated step Model Registration [fe3f1ef7][bace4d1b-0337-45a3-9025-d579dfb3b56e], (This step will run and generate new outputs)\nCreated data reference seer_1740449a for StepId [afded426][a3ea6644-c238-4064-b05b-e1cf3522c3ca], (Consumers of this data will generate new runs.)\nSubmitted PipelineRun d162971e-b6cb-4271-8188-d75b1d1c74ed\nLink to Azure Machine Learning studio: https://ml.azure.com/experiments/seer/runs/d162971e-b6cb-4271-8188-d75b1d1c74ed?wsid=/subscriptions/91d27443-f037-45d9-bb0c-428256992df6/resourcegroups/robots/workspaces/hal\n"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff18968095da4065842741a72d21ae07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": "_PipelineWidget(widget_settings={'childWidgetDisplay': 'popup', 'send_telemetry': False, 'log_level': 'INFO', …"
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/aml.mini.widget.v1": "{\"loading\": true}"
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Submit the pipeline to be run\n",
    "pipeline_run1 = Experiment(ws, 'seer').submit(pipeline1)\n",
    "RunDetails(pipeline_run1).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pipeline_run1.cancel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "published_pipeline1 = pipeline1.publish(\n",
    "    name=\"Food Pipeline\", \n",
    "    description=\"Transfer learned image classifier. Uses folders as labels.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}